﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Web.Mvc;
using NewsDiscoveryEngine;
using NewsMine.DomainObjects;
using NewsMine.Logging;
using NewsMine.Storage;
using NewsMine.DataManagers;
using System.Text.RegularExpressions;
using System.Net;
using System.IO;
using System.Web.Caching;
using WebApp1.Models;
using System.Text;
using NewsMine.Utilities;
using NewsMine.Storage.SQLite;
using System.Collections.Concurrent;
using System.Threading;
using NewsMine.Configuration;

namespace WebApp1.Controllers
{
    public enum FeedGroupType
    {
        NoImageAndMixedDomains = 0,
        ByDomain = 1,
        ByTopic = 2,
        ByCategory = 3,
        ByImageSize = 4,
        BySameImage = 5
    }

    public class FeedGroup
    {
        public DateTime Date { set; get; }
        public string Domain { set; get; }
        public string ImageUrl { set; get; }
        public int ImageWidth { set; get; }
        public int ImageHeight { set; get; }
        public FeedGroupType Type { set; get; }
        public List<FeedItem> Feeds { get; set; }

        public double Score { get; set; }
    }

    public class HomeController : BaseController
    {
        public static ConcurrentDictionary<string,FeedItem> CachedFeedItems = new ConcurrentDictionary<string,FeedItem>();

        string selectedDomain = null;
        int recsPerPage = 60;
        //string gridViewPath = "~/views/home/PinStyle2.cshtml";
        string gridViewPath = "~/views/home/PinStyle2Groups.cshtml";
        string gridViewPathForAdminOptions = "~/views/home/PinStyle2.cshtml";

        //string sharedCurrentUrl = string.Empty;

        QResult currentQResult = new QResult();

        FeedFilterName currentFilter = FeedFilterName.ByDefault;

        public bool IsImageOnlyDisplayEabled { get; set; }

        public bool IsImageDisplayEnabled { get; set; }


        protected static INewsMineLogger _logger = NewsMineLogManager.GetLogger(typeof(HomeController));

        public HomeController()
        {
            IsImageOnlyDisplayEabled = true;
            IsImageDisplayEnabled = true;

            if (NewsMine.Configuration.ConfigurationManager.Get("isOnlyImageDisplayEnabled") != null)
            {
                IsImageOnlyDisplayEabled = Convert.ToBoolean(NewsMine.Configuration.ConfigurationManager.Get("isOnlyImageDisplayEnabled"));
            }


            if (NewsMine.Configuration.ConfigurationManager.Get("isImageDisplayEnabled") != null)
            {
                IsImageDisplayEnabled = Convert.ToBoolean(NewsMine.Configuration.ConfigurationManager.Get("isImageDisplayEnabled"));
            }

            if (NewsMine.Configuration.ConfigurationManager.Get("recsPerPage") != null)
            {
                recsPerPage = Convert.ToInt32(NewsMine.Configuration.ConfigurationManager.Get("recsPerPage"));
            }


        }




        public ActionResult Index(string id)
        {
            // return View(new customView());
            return View("~/views/home/Search.cshtml");
            //return GetView("~/views/home/Index.cshtml", id);
        }




        //
        // GET: /Sort/


        [OutputCache(CacheProfile = "MyProfile")]
        public ActionResult Source(string id, string page)
        {
            currentQResult.CurrentUrl = "/Source/" + id;
            selectedDomain = id;
            currentQResult.IsDisableGrouping = true;
            if (string.IsNullOrEmpty(id))
            {
                selectedDomain = "invalidDomain";
            }

            var domains = NewsMine.DataManagers.WebSiteInfoManager.GetDomainNamesByDomainEndsWith(selectedDomain);

            string searchQuery = "domain:" + selectedDomain + " OR domain:\"" + selectedDomain.Replace(".", " ") + "\"";

            for (int i = 0; i < domains.Count; i++)
            {
                if (domains[i].ToLower() == selectedDomain.ToLower())
                {
                    continue;
                }

                searchQuery = searchQuery + " OR domain:" + domains[i] + " OR domain:\"" + domains[i] + "\"";
            }

            return GetView(gridViewPath, searchQuery, GetPageNumberFromPageName(page));
        }

        [OutputCache(CacheProfile = "MyProfile")]
        public ActionResult Language(string id, string page)
        {
            if (string.IsNullOrEmpty(id))
                id = "English";

            id = RemoveHtmlExtension(id);

            id = id.ToLower();
            currentQResult.CurrentUrl = "/Language/" + id;
            currentQResult.CurrentLanguage = id;

            ChannelManager chMgr = new ChannelManager();
            var ch = ChannelManager.GetDefaultChannelForLanguage(id);

            if (ch != null)
            {
                return Topic(ch.Name, page);
            }


            currentQResult.FeedFilterName = FeedFilterName.language;
            currentQResult.CurrentChannel = ChannelManager.GetDefaultChannelForLanguage(id);

            var viewResult = GetView(gridViewPath, "language:" + id.Trim(), GetPageNumberFromPageName(page));

            return viewResult;
        }

        private string RemoveHtmlExtension(string id)
        {
            if (!string.IsNullOrEmpty(id) && id.ToLower().EndsWith(".html"))
            {
                return id.ToLower().Substring(0, id.Length - 5);
            }
            else
                return id;
        }

        private int GetPageNumberFromPageName(string page)
        {

            if (NewsMine.Utilities.Extensions.IsNumeric(page) == true)
            {
                return Convert.ToInt32(page);
            }
            else
            {
                //hope this will contain the .html extension, so lets split and take first one as number
                string[] segemnts = page.Split('.');

                if (segemnts.Length == 0)
                    return 1;

                if (NewsMine.Utilities.Extensions.IsNumeric(segemnts[0]))
                {
                    return Convert.ToInt32(segemnts[0]);
                }
                else
                    return 1;
            }
        }



        [OutputCache(CacheProfile = "MyProfile")]
        public ActionResult Category(string id, string page)
        {
            if (!string.IsNullOrEmpty(id))
            {
                id = id.ToLower();
            }
            else
            {
                //if id is null or empty. it is an invalid entry.
                Response.Redirect("http://www.lookatnews.com/?ref=noCategoryFound");
            }


            currentQResult.CurrentUrl = "/Category/" + id;

            string language = null;

            var words = id.Split('-');

            words = (from w in words
                     where !string.IsNullOrEmpty(w)
                     select w.Replace("!",string.Empty)).ToList().ToArray();

            if (words.Length > 1)
            {
                language = words[0];
                currentQResult.CurrentLanguage = language;
                id = id.Substring(currentQResult.CurrentLanguage.Length + 1, id.Length - currentQResult.CurrentLanguage.Length - 1);
                currentQResult.CurrentCategory = id;
            }

            if (words.Length == 1)
            {
                currentQResult.CurrentCategory = id;
            }

            if (string.IsNullOrEmpty(language))
            {
                return GetView(gridViewPath, "tags:" + id.Trim(), GetPageNumberFromPageName(page));
            }
            else
                return GetView(gridViewPath, "tags:" + id.Trim() + " AND language:" + language, GetPageNumberFromPageName(page));
        }

        [OutputCache(CacheProfile = "MyProfile")]
        public ActionResult Topic(string id, string page)
        {

            id = RemoveHtmlExtension(id);

            if (string.IsNullOrEmpty(id))
                id = string.Empty;

            id = HttpUtility.UrlDecode(id);

            currentQResult.CurrentUrl = "/topic/" + HttpUtility.UrlEncode(id.Replace("+", " ")); ;

            string searchQuery = string.Empty;

            #region if_channel_exists

            ChannelManager chManager = new ChannelManager();

            if (chManager.IsChannelNameExists(id) == true)
            {
                var selectedChannel = chManager.GetChennelByName(id);

                currentQResult.CurrentChannel = selectedChannel;
                currentQResult.CurrentLanguage = selectedChannel.Language;

                searchQuery = GetSearchQueryForChannel(selectedChannel);

                return GetView(gridViewPath, searchQuery, GetPageNumberFromPageName(page));

            }

            #endregion


            if (!string.IsNullOrEmpty(id) && !id.Contains(":"))
                searchQuery = GetChannelQueryFromString(id);

            else
                searchQuery = id;

            string language = "english";
            var words = id.Split('-');
            if (words.Length > 0)
            {
                language = words[0];
                currentQResult.CurrentLanguage = language;
            }

            if (string.IsNullOrEmpty(searchQuery))
            {
                searchQuery = "language:" + language;
            }
            else
                searchQuery = "(" + searchQuery + ") AND language:" + language;

            return GetView(gridViewPath, searchQuery, GetPageNumberFromPageName(page));

        }

        private string GetSearchQueryForChannel(NewsMine.DomainObjects.Channel selectedChannel)
        {
            return ChannelManager.GetQuery(selectedChannel);
        }





        [OutputCache(CacheProfile = "MyProfile")]
        public ActionResult Default(string id)
        {
            currentQResult.CurrentUrl = "/Language/" + "english";
            currentQResult.CurrentLanguage = "english";

            //return GetView(gridViewPath, "language:" + currentQResult.CurrentLanguage.Trim(), GetPageNumberFromPageName(page));

            //return GetView("~/views/home/pinstyle2.cshtml", id, GetPageNumberFromPageName(page));
            return View("~/views/home/Search.cshtml");
        }

        //RssSearch.cshtml

        public ActionResult RssSearch()
        {
            return View();
        }

        [AdminAuthorizeAttribute]
        public ActionResult AdminSearch(string q, string page)
        {
            currentQResult.FeedFilterName = FeedFilterName.search;
            string searchQuery = q;
            if (string.IsNullOrEmpty(q))
            {
                searchQuery = "*:*";
            }
            currentQResult.SearchString = q;

            currentQResult.IsDisableGrouping = true;

            q = HttpUtility.UrlDecode(q);

            // sharedCurrentUrl = "/AdminSearch/" + HttpUtility.UrlEncode(q);
            currentQResult.CurrentUrl = "/AdminSearch/" + HttpUtility.UrlEncode(q).Replace("+", "%20");

            return GetView(gridViewPath, searchQuery, GetPageNumberFromPageName(page), false);
        }

        public ActionResult Search(string q, string page)
        {
            currentQResult.FeedFilterName = FeedFilterName.search;
            currentQResult.SearchString = q;
            currentQResult.IsDisableGrouping = true;

            if (string.IsNullOrEmpty(q))
            {
                    return Default("");
            }

            q = HttpUtility.UrlDecode(q);

            q = q.Trim();

            //make sure query is lower case
            q = q.ToLower();

            string searchQuery = string.Empty;

            q = NewsMine.Utilities.TempAnalyzer.RemoveAllSpecialCharacterForIndexing(q);

            currentQResult.CurrentUrl = "/Search/" + HttpUtility.UrlEncode(q).Replace("+", "%20");



            searchQuery = GetSearchQueryFromString(q);

            //if (!string.IsNullOrEmpty(q) && !q.Contains(":"))
            //    searchQuery = GetSearchQueryFromString(q);

            //else
            //    searchQuery = q;

            return GetView(gridViewPath, searchQuery, GetPageNumberFromPageName(page), false);
        }

        private string GetSearchQueryFromString(string query)
        {

            query = NewsMine.Utilities.Extensions.RemoveExtraSpaces(query);
            string searchQuery = string.Empty;

            string fieldsList = "title,refurl,link,description,category,language,tags,domain";

            fieldsList = "title,link";

            string[] fields = fieldsList.Split(',');

            fields = GetSimilarWordsFromDictionary(fields);

            string[] words = query.Split(' ', '+', ',', '-');

            foreach (var fld in fields)
            {
                if (words.Length == 1)
                {
                    searchQuery += "(+" + fld + ":" + query + ")^9 OR ";
                    //searchQuery += "(" + fld + ":" + query + ")^8 OR ";
                    //searchQuery += "(" + fld + ":" + query + "?)^9 OR ";
                    searchQuery += "(" + fld + ":" + query + "*)^8 OR ";
                    //searchQuery += "(" + fld + ":" + query + "~0.8) OR ";
                    //searchQuery += "(" + fld + ":" + query + "~0.5) OR ";
                }
                else
                {
                    searchQuery += "(" + fld + ":\"" + query + "\")^9 OR ";
                    searchQuery += "(" + fld + ":\"" + query + "\"~5)^8 OR ";

                    searchQuery += "(" + fld + ":" + query + ")^7 OR ";

                    //searchQuery += "(" + fld + ":\"" + query + "\"~10)^7 OR ";
                    //searchQuery += "(" + fld + ":\"" + query + "\"~0.8)^6 OR ";
                    //searchQuery += "(" + fld + ":\"" + query + "\"~0.5)^5 OR ";

                }


                if (words.Length > 1)
                {
                    //searchQuery += "(" + fld + ":" + query + ")^5 OR ";

                    //string tempQuery = string.Empty;

                    //foreach (var word in words)
                    //{
                    //    if (word.Length > 1)
                    //        tempQuery += "+" + fld + ":" + word + "~0.7";
                    //}

                    //if (!string.IsNullOrEmpty(tempQuery))
                    //    searchQuery += "(" + tempQuery + ")^3 OR ";

                }
            }




            //searchQuery = searchQuery + "(title:" + query.Replace(" ","? ") + ") OR ";
            //searchQuery = "(title:\"" + query + "\"~10^88) OR ";

            //for (int i = 0; i < fields.Length; i++)
            //{
            //    searchQuery = searchQuery + " (" + fields[i] + ":" + query + "*)^" + (9 - i).ToString() + " OR " + " (" + fields[i] + ":" + query + "~)^" + (9 - (i + 1) * 2).ToString() + " OR ";
            //}

            //foreach (string word in words)
            //{
            //    for (int i = 0; i < fields.Length; i++)
            //    {
            //        searchQuery = searchQuery + fields[i] + ":" + word + "* OR ";
            //    }
            //}

            searchQuery = searchQuery.TrimEnd(' ');

            if (searchQuery.EndsWith("OR"))
                searchQuery = searchQuery.Substring(0, searchQuery.Length - 2);

            return searchQuery;
        }

        public ActionResult Go(string id)
        {
            //this is meant for indicating that perticular domain will not work with iframe, so that we will directly give redirect.

            string ni = HttpUtility.UrlDecode(this.Request.QueryString.Get("ni"));

            if (!string.IsNullOrEmpty(ni) && ni.ToLower() == "y")
            {

                string feedid = this.Request.QueryString.Get("id");

                var feedItem = NewsDiscoveryEngine.FeedIndexSearcher.Instance.GetFeedById(Convert.ToInt64(feedid));

                if (feedItem == null)
                {
                    if (this.Request.UrlReferrer != null)
                    {
                        Response.Redirect(this.Request.UrlReferrer.ToString());
                    }
                    else
                    {
                        Response.Redirect("http://www.lookatnews.com/?ref=nofeeditem");
                    }
                }

                string url = feedItem.Link;

                // string url = HttpUtility.UrlDecode(this.Request.QueryString.Get("u"));

                if (string.IsNullOrEmpty(url))
                {
                    Response.Write("<h1>Invalid url found... Sorry for this. We will try to fix this issue as soon as possible.</h1>");
                }

                return Redirect(url);
            }
            else
            {
                //return PartialView("~/views/shared/_goto.cshtml");


                return View("~/views/shared/_goto.cshtml");
            }
        }

        [AdminAuthorizeAttribute]
        public ActionResult fb(string id)
        {
            bool isAdminPortal = Convert.ToBoolean(System.Configuration.ConfigurationManager.AppSettings.Get("IsAdminPortal"));

            if (isAdminPortal == false)
                return Content("This method is allowed only in the admin portal mode.");

            string feedId = HttpUtility.UrlDecode(this.Request.QueryString.Get("feedId"));
            string fbpageid = HttpUtility.UrlDecode(this.Request.QueryString.Get("fbpageid"));



            if (string.IsNullOrEmpty(fbpageid))
            {
                return PartialView("~/views/shared/_fbPost.cshtml");
            }

            if (!string.IsNullOrEmpty(feedId) && !string.IsNullOrEmpty(fbpageid))
            {
                feedId = feedId.Trim();

                var fbPageList = NewsMine.DataManagers.FaceBookPageManager.GetFacebookPageInfo();

                var selectedFaceBookPage = (from fbpage in fbPageList
                                            where fbpage.Id.ToLower() == fbpageid.ToLower()
                                            select fbpage).FirstOrDefault<FacebookPage>();

                PrepareFacebookPostDataFromUserSelectionJob job = new PrepareFacebookPostDataFromUserSelectionJob();

                job.FacebookEmailPostInputData = new FaceBookEmailPostData
                    {
                        FeedId = Convert.ToInt64(feedId),
                        SelectedFacebookPage = selectedFaceBookPage
                    };


                Defaults.FaceBookDownloadingJobQueue.Enqueue(job);

            }

            return Content("Items is successfully submitted to publish in facebook.");
        }


        public ActionResult GetView(string viewPath)
        {
            string domain = null;

            domain = this.Request.QueryString.Get("domain");

            if (domain == null)
                domain = "telugu.oneindia.in";

            return GetView(viewPath, "source:" + domain);

        }

        public ActionResult GetView(string viewPath, string searchQuery)
        {
            return GetView(viewPath, searchQuery, 1);
        }

        public ActionResult GetView(string viewPath, string searchQuery, int page)
        {
            return GetView(viewPath, searchQuery, page, true);
        }

        public ActionResult GetView(string viewPath, string searchQuery, int page, bool orderByLatest)
        {
            if (page < 1)
                page = 1;

            if (string.IsNullOrEmpty(searchQuery) || string.IsNullOrEmpty(searchQuery.Trim()))
                return GetView(viewPath);

            string currentUrl = this.Request.Url.AbsolutePath;
            currentUrl = currentQResult.CurrentUrl;

            bool isDebugMode = Convert.ToBoolean(NewsMine.Configuration.ConfigurationManager.Get("isDebugMode"));

            if (!isDebugMode && !Request.QueryString.AllKeys.Contains("refreshbot") && currentQResult.FeedFilterName != FeedFilterName.search)
            {
                currentQResult.IsDisableGrouping = true;

                try
                {
                    string keyUrl = Request.Url.Scheme + "://" + Request.Url.Host + currentQResult.CurrentUrl + "/" + page.ToString();
                    keyUrl = keyUrl.ToLower();

                    object htmlContentObject = CacheAgainstUrlHelper.Get(keyUrl);

                    if (htmlContentObject != null && htmlContentObject is CachedHtmlContent)
                    {
                        CachedHtmlContent cachedHtmlContent = (CachedHtmlContent)htmlContentObject;

                        if (!UrlsToRefresh.Contains(keyUrl) && cachedHtmlContent.CreatedTime.AddMinutes(10) < DateTime.Now)
                        {
                            UrlsToRefresh.Enqueue(keyUrl);
                            AddTaskTrigger();
                        }


                        //Response.Headers.Add("fromStore", "y"); // this is just for testing...

                        //return Content(cachedHtmlContent.Html);

                        if (cachedHtmlContent.CreatedTime.AddMinutes(60) >= DateTime.Now &&  !string.IsNullOrEmpty(cachedHtmlContent.Html))
                        {
                            Response.Headers.Add("fromStore", "y"); // this is just for testing...

                            return Content(cachedHtmlContent.Html);
                        }
                        
                    }
                    else
                    {
                        if (!UrlsToRefresh.Contains(keyUrl))
                        {
                            UrlsToRefresh.Enqueue(keyUrl);
                            AddTaskTrigger();
                        }
                    }

                    
                }
                catch (Exception exx)
                {
                    TempLogger.Log("Error while getting cached html from the store", exx);
                }
            }



            //currentUrl = currentUrl + "?id=" + searchQuery;



            List<WebSiteInfo> webSites;
            List<FeedItem> feeds;

            int pageNumber = page;

            string domain = null;

            domain = selectedDomain;


          

            int totalRecs;

            if (IsImageOnlyDisplayEabled == true)
            {
                searchQuery = "(" + searchQuery + ") AND imageurl:http*";
            }
            //else
            //{
            //    searchQuery = "(" + searchQuery + ") OR imageurl:http*^9";
            //}

            //todo: needs to decide the ranking for non image items
            //searchQuery = "(" + searchQuery + ") AND imageurl:h*";

            int tempPageNumber = pageNumber - 1;

            GetFeedByDomain(out webSites, out feeds, out totalRecs, ref tempPageNumber, searchQuery, orderByLatest);

            feeds = RemoveUnWantedFeeds(feeds);

            
            if (Request.QueryString.AllKeys.Contains("refreshbot") && currentQResult.CurrentChannel != null && currentQResult.CurrentChannel.IsDefault ==false)
            {
                string[] keywordsInChannelName = currentQResult.CurrentChannel.Name.Split('-').Where(w => !string.IsNullOrEmpty(w) && !string.IsNullOrEmpty(w.Trim())).Select(w =>w.ToLower()).ToArray();

                feeds = (from f in feeds
                         where string.IsNullOrEmpty(f.AnalyzedTag)
                         || keywordsInChannelName.Contains(f.AnalyzedTag.ToLower())
                         select f).ToList();
            }

            foreach (var f in feeds)
            {
                if (!HomeController.CachedFeedItems.ContainsKey(f.ID))
                {
                    HomeController.CachedFeedItems.TryAdd(f.ID, f);
                }
            }


            CreateFeedGroups(feeds);


            //sort the feeds by pubdate, becuase if feeds are only one page, the recent news are going down.

            if (feeds != null && feeds.Count > 0)
            {
                for (int i = 0; i < feeds.Count; i++)
                {
                    if (string.IsNullOrEmpty(feeds[i].Title))
                    {
                        feeds[i].Title = string.Empty;
                    }

                    if (feeds[i].PubDate.AddHours(1) > DateTime.UtcNow)
                    {
                        feeds[i].Rank = 140 - Convert.ToInt32((DateTime.UtcNow - feeds[i].PubDate).TotalMinutes);
                        feeds[i].Rank = feeds[i].Rank / 5;  // making sure with in minutes sorting happens... images gets priority.
                    }

                    //if (feeds[i].PubDate.AddHours(1) > DateTime.UtcNow)
                    //{
                    //    feeds[i].Rank = Convert.ToInt32((DateTime.UtcNow - feeds[i].PubDate).TotalMinutes);
                    //}
                }
            }


            if (feeds != null && feeds.Count > 0)
            {
                feeds = (from feed in feeds
                         orderby feed.Rank descending, Convert.ToInt64((((DateTime.UtcNow - feed.PubDate).TotalHours) / 3)) ascending, feed.ImageWidth descending, feed.Title descending
                         select feed).ToList<FeedItem>();

            }


            if (tempPageNumber != (pageNumber - 1))
            {
                pageNumber = tempPageNumber + 1;
            }

            //feeds = FilterFeedAgainstBlockedStuff(feeds);

            int totalNumberOfPages = GetTotalNumberOfPages(totalRecs);
            WebSiteInfo currentDomain = null;

            if (string.IsNullOrEmpty(domain) == false)
            {
                currentDomain = (from wb in webSites
                                 where wb.Domain != null && wb.Domain.ToLower() == domain.ToLower()
                                 select wb).FirstOrDefault<WebSiteInfo>();
            }


            if (string.IsNullOrEmpty(currentQResult.CurrentLanguage))
                currentQResult.CurrentLanguage = "all";

            if (currentDomain != null && !string.IsNullOrEmpty(currentDomain.Language))
            {

                currentQResult.CurrentLanguage = currentDomain.Language;
            }

            //its important to have majority feeds language as current language.

            if (feeds.Count > 0 && (currentQResult.CurrentLanguage != null && currentQResult.CurrentLanguage.ToLower() != "marathi"))
            {
                currentQResult.CurrentLanguage = DetectLanguageFromFilteredFeeds(feeds);
            }

            var languageSpecificWebSites = (from wb in webSites
                                            where wb.Language != null && wb.Language.ToLower() == currentQResult.CurrentLanguage.ToLower()
                                            select wb).ToList<WebSiteInfo>();

            if (currentQResult.CurrentLanguage == "all")
                languageSpecificWebSites = webSites;

            var pager = Pager.Items(totalNumberOfPages * recsPerPage).PerPage(recsPerPage).Move(pageNumber).Segment(5).Center();

            ViewBag.Pager = pager;


            bool IsAdminPortal = Convert.ToBoolean(System.Configuration.ConfigurationManager.AppSettings.Get("IsAdminPortal"));
            //string adminview = this.Request.Cookies.Get("adminview");

            if (IsAdminPortal == true)
            {
                viewPath = "~/views/home/PinStyle2.cshtml";
            }

            //testing...
            //viewPath = gridViewPathForAdminOptions;

            var v = View(viewPath,
                new QResult
                {
                    IsDisableGrouping = currentQResult.IsDisableGrouping,
                    CurrentChannel = currentQResult.CurrentChannel,
                    SearchString = currentQResult.SearchString,
                    IsImageDisplayEnabled = this.IsImageDisplayEnabled,
                    CurrentCategory = currentQResult.CurrentCategory,
                    CurrentLanguage = currentQResult.CurrentLanguage,
                    CurrentDomain = currentDomain,
                    Websites = webSites,
                    FeedItems = feeds,
                    PageNumber = pageNumber,
                    CurrentUrl = currentUrl,
                    CurrentLanguageWebsites = languageSpecificWebSites,
                    TotalNumberPages = totalNumberOfPages,
                    Pager = pager
                });

            return v;
        }

        #region windows service kind of stuff.

        bool IsCacheUpdateIsEnabled
        {
            get
            {
                var cfgValue = NewsMine.Configuration.ConfigurationManager.Get("IsCacheUpdateIsEnabled");
                if (string.IsNullOrEmpty(cfgValue))
                    return true;
                else
                    return Convert.ToBoolean(cfgValue);
            }
        }



        private static CacheItemRemovedCallback OnCacheRemove = null;

        //AddTask("DoStuff", 60);
        private void AddTaskTrigger()
        {
            if (HttpContext.Cache.Get("DoStuff") == null)
                AddTask("DoStuff", 30);
        }

        DateTime lastTimeTaskAdded = DateTime.MinValue;

        private void AddTask(string name, int seconds)
        {
            if (lastTimeTaskAdded.AddSeconds(20) > DateTime.UtcNow)
                return;

            lastTimeTaskAdded = DateTime.UtcNow;

            OnCacheRemove = new CacheItemRemovedCallback(CacheItemRemoved);
            HttpRuntime.Cache.Insert(name, seconds, null,
                DateTime.Now.AddSeconds(seconds), Cache.NoSlidingExpiration,
                CacheItemPriority.NotRemovable, OnCacheRemove);
        }

        static ConcurrentQueue<string> UrlsToRefresh = new ConcurrentQueue<string>();

        public void CacheItemRemoved(string k, object v, CacheItemRemovedReason r)
        {
            if (IsCacheUpdateIsEnabled == false)
                return;

            // do stuff here if it matches our taskname, like WebRequest
            string cacheBreakingUrl = string.Empty;
            string url = string.Empty;

            string html = string.Empty;

            DateTime startTime = DateTime.MinValue;
            DateTime endTime = DateTime.MinValue;

            while (HomeController.UrlsToRefresh.Count > 0)
            {
                try
                {

                    if (UrlsToRefresh.Count == 0)
                    {
                        Thread.Sleep(500);
                        continue;
                    }
                    url = string.Empty;
                    UrlsToRefresh.TryDequeue(out url);

                    if (string.IsNullOrEmpty(url))
                    {
                        Thread.Sleep(100);
                        continue;
                    }

                    cacheBreakingUrl = string.Empty;

                    if (url.Contains("?"))
                    {
                        cacheBreakingUrl = url + "&refreshbot=true";
                    }
                    else
                        cacheBreakingUrl = url + "?refreshbot=true";

                    html = string.Empty;

                    startTime = DateTime.UtcNow;

                    html = NewsMine.Utilities.WebHelper.DownloadHtmlFromUrl(cacheBreakingUrl);

                    endTime = DateTime.UtcNow;

                    System.IO.File.AppendAllText("C:\\refreshUrls.txt", Environment.NewLine + DateTime.Now.ToString() + " - Time " + (endTime-startTime).TotalMilliseconds.ToString() + " ms." + " - " + url );


                    CacheAgainstUrlHelper.Replace(url.ToLower(), new CachedHtmlContent { Html = html, CreatedTime = DateTime.UtcNow });

                }
                catch (Exception ex)
                {
                    TempLogger.Log("Error while refreshing html content for caching in the CacheItemRemoved method.", ex);
                }
            }



            // re-add our task so it recurs
            AddTask(k, Convert.ToInt32(v));
        }


        #endregion



        private List<FeedItem> RemoveUnWantedFeeds(List<FeedItem> feeds)
        {
            string tempTitle = string.Empty;
            List<FeedItem> filteredFeeds = new List<FeedItem>();

            for (int i = 0; i < feeds.Count; i++)
            {
                string truncatedTitle = feeds[i].Title;

                if (string.IsNullOrEmpty(truncatedTitle))
                    continue;

                // this is temp fix, for one of the malayalam channel.
                if (truncatedTitle.Contains("<!--") || truncatedTitle.Contains(".popover();")
                    || truncatedTitle.StartsWith("आगे »")
                    )
                {
                    continue;
                }

                filteredFeeds.Add(feeds[i]);

            }

            return filteredFeeds;
        }

        #region Grouping stuff



        private void CreateFeedGroups(List<FeedItem> feeds)
        {

            return;
            //var titles = (from fd in feeds
            //              select new
            //              {
            //                  FeedItem = fd,
            //                  SimilarDocs = feeds.Select(f => NewsMine.Utilities.FuzzySearch.Search(f.Title, feeds.Select(ff => ff.Title).ToList(), 0.70))
            //              }).ToList();


            foreach (var fd in feeds)
            {
                Response.Write("<b>" + fd.Title + "</b><br/>");
                var compareList = (from ff in feeds where ff.Title != fd.Title select ff).ToList();

                Response.Write("<table>");

                foreach (var compareStr in compareList)
                {
                    //var similarity = NewsMine.Utilities.FuzzySearch.LevenshteinDistanceWordLevel(fd.Title, compareStr);
                    var similarity = NewsMine.Utilities.FuzzySearch.SimilarityOfTwoStrings(fd.Title, compareStr.Title);

                    if (similarity < 50)
                        continue;

                    Response.Write("<tr>");
                    Response.Write("<td><b>");
                    Response.Write(fd.Title);
                    Response.Write("</b></td>");
                    Response.Write("<td>");
                    Response.Write(compareStr.Title);
                    Response.Write("</td>");

                    Response.Write("<td>");
                    Response.Write(similarity.ToString());
                    Response.Write("</td>");

                    Response.Write("</tr>");
                }
                Response.Write("</table>");


                //var similar = NewsMine.Utilities.FuzzySearch.SearchByWordLevel(fd.Title, compareList, 0.50);

                //Response.Write("Similar docs<br/>");


                //if (similar.Count > 1)
                //{
                //    foreach (var s in similar)
                //    {
                //        Response.Write("<i>" + s + "</i><br/>");

                //    }


                //}

            }

        }

        #endregion




        #region Helper Methods

        private int GetTotalNumberOfPages(int totalRecs)
        {
            //if records 100, then number of pages 30x3 + 10 =100

            if (totalRecs % recsPerPage > 0)
                return totalRecs / recsPerPage + 1;
            else
                return totalRecs / recsPerPage;

        }



        private string GetChannelQueryFromString(string id)
        {

            string searchQuery = string.Empty;

            string fieldsList = "title,refurl,link,category,tags";

            string[] fields = fieldsList.Split(',');



            string[] words = id.Split(' ', '+', ',', '-');

            words = GetSimilarWordsFromDictionary(words);

            for (int wordCount = 0; wordCount < words.Length; wordCount++)
            {
                var word = words[wordCount];
                for (int i = 0; i < fields.Length; i++)
                {
                    searchQuery = searchQuery + fields[i] + ":" + word + "^" + (words.Length - wordCount).ToString() + " OR ";
                }
            }

            if (string.IsNullOrEmpty(searchQuery))
                return null;

            searchQuery = searchQuery.Substring(0, searchQuery.Length - 3);

            return searchQuery;
        }

        private string[] GetSimilarWordsFromDictionary(string[] fields)
        {
            //needs to be implemented.
            List<string> resultedFields = new List<string>();

            foreach (var f in fields)
            {
                if (IsThisLanguageTerm(f))
                    continue;
                else
                    resultedFields.Add(f);
            }

            return resultedFields.ToArray();
        }

        private bool IsThisLanguageTerm(string f)
        {
            var languages = WebApp1.Models.LanguageModel.GetAllLanguages();

            foreach (var l in languages)
            {
                if (l.Name.ToLower() == f.ToLower())
                    return true;
            }

            return false;
        }


        private void GetFeedByDomain(out List<WebSiteInfo> webSites, out List<FeedItem> feeds, out int totalRecs, ref  int pageNumber, string query, bool orderByLatest)
        {


            var webSiteKVStore = Defaults.WebSiteInfoMasterStore;

            int count = webSiteKVStore.GetCount();

            var websiteObjects = NewsMine.DataManagers.WebSiteInfoManager.GetAllDomains();

            webSites = new List<WebSiteInfo>();

            foreach (var websiteInfoObject in websiteObjects)
            {
                WebSiteInfo wbSiteInfo = (WebSiteInfo)websiteInfoObject;
                if (wbSiteInfo.MenuItemconfigs != null && wbSiteInfo.MenuItemconfigs.Count > 0)
                    webSites.Add(wbSiteInfo);
            }

            webSites = (from wb in webSites
                        orderby wb.Language
                        select wb).ToList<WebSiteInfo>();


            string domain = null;

            domain = this.Request.QueryString.Get("domain");

            if (domain == null)
                domain = "telugu.oneindia.in";

            //todo: some times readerclosed error coming, so just trying again. Just Work around needs to be fixed
            try
            {
                var indxer = FeedIndexSearcher.Instance;

                feeds = indxer.GetFeedItemsByQueryOrderByLatest(query, pageNumber, recsPerPage, out totalRecs, orderByLatest);

                if (feeds.Count == 0)
                {
                    pageNumber = 0;
                    feeds = indxer.GetFeedItemsByQueryOrderByLatest(query, pageNumber, recsPerPage, out totalRecs, orderByLatest);
                }

            }
            catch (Exception ex)
            {
                System.Threading.Thread.Sleep(400);

                var indxer = FeedIndexSearcher.Instance;

                feeds = indxer.GetFeedItemsByQueryOrderByLatest(query, pageNumber, recsPerPage, out totalRecs, orderByLatest);

                if (feeds.Count == 0)
                {
                    pageNumber = 0;
                    feeds = indxer.GetFeedItemsByQueryOrderByLatest(query, pageNumber, recsPerPage, out totalRecs, orderByLatest);
                }
            }
            //if(true)
            //{
            //    long min = 0;
            //    long max = 2000;

            //    feeds = indxer.GetFeedItemsByIDRange(min,max,pageNumber, recsPerPage, out totalRecs);
            //}

            // HttpContext.Response.Write("Total records matched with = " + totalRecs.ToString());

            //feeds = (from fd in feeds
            //         where fd.PubDate.AddDays(365) > DateTime.UtcNow || true
            //         orderby fd.PubDate descending
            //         select fd).ToList<FeedItem>();

            if (feeds == null)
                feeds = new List<FeedItem>();


            //int maxRecords = 100;

            //if (feeds.Count < 100)
            //    maxRecords = feeds.Count;
            //feeds = feeds.GetRange(0, maxRecords);

            //Do temporary cleaning on the images.
            //todo: Just need to move this code into the cleanup module in the job engine.

            for (int i = 0; i < feeds.Count; i++)
            {
                var feed = feeds[i];

                //if (!string.IsNullOrEmpty(feed.ImageUrl) && System.Text.RegularExpressions.Regex.IsMatch(feed.ImageUrl, @".*[.]+\w$"))
                //    feed.ImageUrl = null;

                if (string.IsNullOrEmpty(feed.ImageUrl))
                    continue;
                try
                {
                    //clensing this needs to goto clawler code.

                    feed.ImageUrl = feed.ImageUrl.TrimEnd('\"').TrimEnd('/').TrimEnd('\\');

                    if (feed.ImageUrl.EndsWith("\"\\"))
                    {
                        feed.ImageUrl = feed.ImageUrl.Replace("\"\\", string.Empty);
                    }

                    feed.ImageUrl = feed.ImageUrl.TrimEnd('\"').TrimEnd('/').TrimEnd('\\');

                    Uri uri = new Uri(feed.ImageUrl);
                    string filename = System.IO.Path.GetFileName(uri.LocalPath);
                    string fileExtension = System.IO.Path.GetExtension(uri.LocalPath);
                    if (string.IsNullOrEmpty(filename) || string.IsNullOrEmpty(fileExtension))
                    {
                        feed.ImageUrl = null;
                    }
                }
                catch (Exception ex)
                {
                    //todo: needs to log the exception message here.

                    //if something is happening there in the try block, means of osme issue with image url. 
                    feed.ImageUrl = null;
                }
            }



        }


        private string DetectLanguageFromFilteredFeeds(List<FeedItem> feeds)
        {

            List<string> languagesFound = (from fd in feeds
                                           where string.IsNullOrEmpty(fd.Language) == false
                                           select fd.Language).ToList<string>();


            var lngCount = languagesFound
                 .GroupBy(i => i)
                  .Select(i => new
                  {
                      Name = i.Key,
                      Count = i.Count()
                  }).ToList();

            if (lngCount.Count == 0)
                return null;

            var result = (from l in lngCount
                          orderby l.Count descending
                          select l).FirstOrDefault();

            if (result == null)
                return null;
            else
                return result.Name;

        }




        #endregion



        #region Un used Actions

        public ActionResult Test()
        {
            return View("~/views/shared/bootstraptest.cshtml");
        }

        public ActionResult Channel(string id)
        {
            return GetView("~/views/home/Index.cshtml");
        }


        public void CatchAll(string id)
        {
            //string[] PassedValue = this.Request.Url.AbsolutePath.Split('/');
            //return "You are Passing Values:" + String.Join(",", PassedValue);

            bool? checkUrl = Convert.ToBoolean(this.Request.QueryString.Get("checkurl"));

            string remoteServerHost = NewsMine.Configuration.ConfigurationManager.Get("remoteServer");

            string localSubdir = NewsMine.Configuration.ConfigurationManager.Get("localSubdir");
            string targetUrl = null;

            string currentUrl = this.Request.Url.ToString();

            if (currentUrl.EndsWith("favicon.ico"))
            {
                Response.StatusCode = 404;
                return;
            }

            if (!string.IsNullOrEmpty(localSubdir) && currentUrl.Contains(this.Request.Url.Host + "/" + localSubdir))
            {

                targetUrl = currentUrl.Replace(this.Request.Url.Host + "/" + localSubdir, remoteServerHost);
            }
            else
            {
                targetUrl = currentUrl.Replace(this.Request.Url.Host, remoteServerHost);

            }
            object cachedHtml = HttpContext.Cache.Get(targetUrl);

            if (cachedHtml != null)
            {
                this.Response.Write(cachedHtml);
                return;
            }

            if (checkUrl.HasValue && checkUrl.Value == true)
            {
                Response.Write(targetUrl);
                return;
            }


            string html = NewsMine.Utilities.WebHelper.DownLoadHttpTextByCompressionEnabled(targetUrl);

            //WebRequest request = WebRequest.Create(targetUrl);

            ////request.Headers = this.Request.Headers;


            ////todo: needs to be tweaked and read from the configuration.
            //request.Timeout = 30 * 1000;

            //WebResponse response = request.GetResponse();
            //Stream data = response.GetResponseStream();

            //string html = String.Empty;
            //using (StreamReader sr = new StreamReader(data))
            //{   
            //    html = sr.ReadToEnd();
            //}

            HttpContext.Cache.Insert(targetUrl, html, null, DateTime.Now.AddSeconds(120), Cache.NoSlidingExpiration, CacheItemPriority.Normal, new CacheItemRemovedCallback(CachedItemRemoveCallBack));


            this.Response.Write(html);

        }

        private void CachedItemRemoveCallBack(string key, object value,
                CacheItemRemovedReason reason)
        {

            //for now we will disable this feature...

            return;

            string targetUrl = key;
            WebRequest request = WebRequest.Create(targetUrl);

            //request.Headers = this.Request.Headers;


            //todo: needs to be tweaked and read from the configuration.
            //request.Timeout = 30 * 1000;

            WebResponse response = request.GetResponse();
            Stream data = response.GetResponseStream();

            string html = String.Empty;
            using (StreamReader sr = new StreamReader(data))
            {
                html = sr.ReadToEnd();
            }

            HttpContext.Cache.Insert(targetUrl, html, null, DateTime.Now.AddSeconds(20), Cache.NoSlidingExpiration, CacheItemPriority.Normal, new CacheItemRemovedCallback(CachedItemRemoveCallBack));


        }

        #endregion



    }



}

public enum FeedFilterName
{
    ByDefault,
    language,
    category,
    search,
    Channel,
    source
}

public class QResult
{
    public QResult()
    {
        IsDisableGrouping = false;
    }

    public string SearchString { set; get; }
    public FeedFilterName FeedFilterName { get; set; }
    public List<WebSiteInfo> Websites { get; set; }
    public List<WebSiteInfo> CurrentLanguageWebsites { get; set; }
    public List<FeedItem> FeedItems { get; set; }
    public int PageNumber { get; set; }
    public int TotalNumberPages { get; set; }
    public int TotalRecords { get; set; }
    public string CurrentUrl { get; set; }
    public bool IsDisableGrouping { get; set; }
    public WebSiteInfo CurrentDomain { get; set; }
    public Channel CurrentChannel { get; set; }
    public string CurrentLanguage { get; set; }
    public string CurrentCategory { get; set; }
    public string SelectedDomain { get; set; }
    public bool IsImageDisplayEnabled { get; set; }

    public void SortWebSitesByAlexaRank()
    {
        if (Websites != null && Websites.Count > 0)
        {
            Websites = Websites.OrderBy(o => o.AlexaRank).ToList();
        }

        if (CurrentLanguageWebsites != null && CurrentLanguageWebsites.Count > 0)
        {
            CurrentLanguageWebsites = CurrentLanguageWebsites.OrderBy(o => o.AlexaRank).ToList();
        }
    }


    public string Title { get; set; }
    public string Description { set; get; }


    public void PrepareTitleAndDescription()
    {
        Description = "{c} , {l} News from all regional newspapers and online sources at LookAtNews.com largest indian regional news paper aggregator.";
        if (string.IsNullOrEmpty(Title))
        {
            if (CurrentDomain != null)
            {
                Description = "Top Headlines, Latest News from ";

                if (!string.IsNullOrEmpty(CurrentDomain.Name) && CurrentDomain.Domain != CurrentDomain.Name)
                {
                    Title = Title + CurrentDomain.Name.ToUpperInvariant() + " / ";
                    Description = Description + CurrentDomain.Name.ToUpperInvariant() + " / ";

                }

                if (!string.IsNullOrEmpty(CurrentDomain.LocalName) && CurrentDomain.Name != CurrentDomain.LocalName)
                {
                    Title = Title + CurrentDomain.LocalName + " / ";
                    Description = Description + "(" + CurrentDomain.LocalName + ") -  ";

                }

                if (!string.IsNullOrEmpty(CurrentDomain.Domain))
                {
                    Title = Title + CurrentDomain.Domain;

                    Description = Description + CurrentDomain.Domain;

                }

                Title = Title + " News | LookAtNews.Com";
            }
            else if (!string.IsNullOrEmpty(CurrentLanguage))
            {
                //if (!string.IsNullOrEmpty(CurrentCategory))
                //{
                //    Title = CurrentCategory.ToUpperInvariant() + " News | ";
                //}

                if (!string.IsNullOrEmpty(CurrentCategory))
                {
                    Title = Title + CurrentCategory.ToLower() + " news | ";
                    Description = Description.Replace("{c}", CurrentCategory + " news ");

                }
                else
                {
                    Description = Description.Replace("{c}", string.Empty);
                }

                var languageModel = LanguageModel.GetLanguageByLanguageName(CurrentLanguage);

                if (languageModel != null)
                {
                    Title = Title + languageModel.Name + " News |" + "(Local: " + languageModel.LocalName + " News) |";
                    Description = Description.Replace("{l}", languageModel.Name + " (Local: " + languageModel.LocalName + ") Language ");

                }

                Title = Title + " LookAtNews.Com";
            }



            //Description = string.Format(Description, "Indan Latest and Breaking News ");

            //Title = Title + " India News Headlines | LookAtNews.Com";


            if (PageNumber > 1)
            {
                Title = Title + " Page " + PageNumber.ToString();
            }

            Description = NewsMine.Utilities.Extensions.RemoveExtraSpaces(Description.Trim());
            Title = NewsMine.Utilities.Extensions.RemoveExtraSpaces(Title.Trim());

            Title = System.Threading.Thread.CurrentThread.CurrentCulture.TextInfo.ToTitleCase(Title);
            Description = System.Threading.Thread.CurrentThread.CurrentCulture.TextInfo.ToTitleCase(Description);

        }


    }

    public Pager Pager { get; set; }

}


